package AGENTS;

import Bots.Buffer;
import cz.cuni.amis.utils.exception.PogamutException;

import java.io.*;

public class AGENTS_ART {


    //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    //构造
    static int capacity = 9999;
    static int WriteInTarget = 50;
    static int K = 16;

    public int numState;
    public int numAction;
    public int numReward = 2;

    public final int numSpace = 4;
    public final int CURSTATE = 0;
    public final int ACTION = 1;
    public final int REWARD = 2;
    public final int NEXTSTATE = 3;

    public double[][] I;

    private double[] T;
    public int[] numInput;

    public int numCode = 0;
    public double[][][] W;
    public int[][] N;

    public int target_numCode = 0;
    public double[][][] target_W;


    private boolean[] newCode;

    private int[] y;


    public AGENTS_ART(int agent_num, int numState, int numAction, boolean rw) throws IOException {

        this.numState = numState;
        this.numAction = numAction;
        nnet = new NN(numState, numAction);

        numInput = new int[numSpace];   //// numSpace:0-State 1-Action 2-Reward 
        numInput[0] = numState;
        numInput[1] = numAction;
        numInput[2] = numReward;
        numInput[3] = numState;

        I = new double[numSpace][];
        for (int i = 0; i < numSpace; i++)
            I[i] = new double[numInput[i]];


        T = new double[capacity];
        N = new int[capacity][];

        if (!rw) {
            numCode = 0;
            target_numCode = 0;
            newCode = new boolean[capacity];
            y = new int[capacity];

            newCode[numCode] = true;
            y[numCode] = 0;


            W = new double[capacity][][];
            target_W = new double[capacity][][];

            for (int j = 0; j < numCode + 1; j++) {
                W[j] = new double[numSpace][];
                N[j] = new int[numAction];
                target_W[j] = new double[numSpace][];
                for (int k = 0; k < numSpace; k++) {
                    W[j][k] = new double[numInput[k]];
                    target_W[j][k] = new double[numInput[k]];
                    for (int i = 0; i < numInput[k]; i++) {
                        W[j][k][i] = (double) 1.0;
                        target_W[j][k][i] = (double) 1.0;

                    }
                }
            }
        } else {
            try {
                readfile(agent_num);
            } catch (Exception e) {
                System.out.println("error");
                numCode = 0;
                target_numCode = 0;
                newCode = new boolean[capacity];
                y = new int[capacity];

                newCode[0] = true;
                y[0] = 0;


                T = new double[capacity];

                W = new double[capacity][][];
                target_W = new double[capacity][][];

                for (int j = 0; j < numCode + 1; j++) {
                    W[j] = new double[numSpace][];
                    N[j] = new int[numAction];
                    target_W[j] = new double[numSpace][];
                    for (int k = 0; k < numSpace; k++) {
                        W[j][k] = new double[numInput[k]];
                        target_W[j][k] = new double[numInput[k]];
                        for (int i = 0; i < numInput[k]; i++) {
                            W[j][k][i] = (double) 1.0;
                            target_W[j][k][i] = (double) 1.0;

                        }
                    }
                }
                ;
            }
            System.out.println("numCode");
            System.out.println(numCode);
        }
    }


//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////


    //提取环境信息
    public void setState(double[] stat) {//提取状态
        for (int i = 0; i < numState; i++) {
            I[CURSTATE][i] = stat[i];
        }
    }

    public void actiontoact(int action, double[] act) {
        for (int i = 0; i < numAction; i++)
            act[i] = 0;
        act[action] = 1;
    }

    public void setAction(double[] act) {//提取动作
        for (int i = 0; i < numInput[ACTION]; i++)
            I[ACTION][i] = act[i];
    }

    public void setReward(double r) {//提取奖励
        I[REWARD][0] = r;
        I[REWARD][1] = 1 - r;
    }

    public void setNextState(double[] stat) {//提取状态
        for (int i = 0; i < numState; i++) {
            I[NEXTSTATE][i] = stat[i];
        }
    }


////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////


    //表现部分

    public void Nx(int num, double[] x) {
        double sum = 0.0;
        for (int i = 0; i < num; i++) {
            sum += Math.pow(x[i], 2);
        }
        sum = Math.pow(sum, 1.0 / 2.0);
        for (int i = 0; i < num; i++) {
            x[i] = x[i] / sum;
        }
    }

    public double distance(int num, double[] x) {
        double sum = 0.0;
        for (int i = 0; i < num; i++) {
            sum += Math.pow(x[i], 2);
        }
        sum = Math.pow(sum, 1.0 / 2.0);
        return sum;
    }


    //代码激活（F2节点思考T--activityF2）

    public double alpha[] = {(double) 0.1, (double) 0.1, (double) 0.1};
    public double gamma[] = {(double) 1.0, (double) 1.0, (double) 1.0};
    int learn_func = 3;


    public void CodeActivition(int numSpace) {
        if (learn_func == 0) {
            //1.ART2-A
            for (int j = 0; j < numCode + 1; j++) {//激活函数
                T[j] = 0;
                for (int k = 0; k < numSpace; k++) {
                    double sum = 0.0;
                    for (int i = 0; i < numInput[k]; i++) {
                        sum += Math.pow(I[k][i], 2);
                    }
                    sum = Math.pow(sum, 1.0 / 2.0);

                    double sum2 = 0.0;
                    for (int i = 0; i < numInput[k]; i++) {
                        if (I[k][i] <= alpha[k]) {
                            I[k][i] = 0;
                        } else {
                            I[k][i] /= sum;
                        }
                        sum2 += I[k][i];
                    }
                    sum2 = Math.pow(sum2, 1.0 / 2.0);

                    for (int i = 0; i < numInput[k]; i++) {
                        I[k][i] /= sum2;
                        if (newCode[j]) {
                            T[j] += alpha[k] * I[k][i];
                        } else {
                            T[j] += gamma[k] * I[k][i] * W[j][k][i];
                        }
                    }
                }
            }
        } else if (learn_func == 1) {
            //2.FuzzyART
            double top, bottom;
            for (int j = 0; j < numCode + 1; j++) {//激活函数
                T[j] = 0;
                for (int k = 0; k < numSpace; k++) {
                    top = 0;
                    bottom = (double) alpha[k];
                    for (int i = 0; i < numInput[k]; i++) { //(I v W) / W
                        top += Math.min(I[k][i], W[j][k][i]);
                        bottom += W[j][k][i];
                    }
                    T[j] += gamma[k] * (double) (top / bottom); //Tj
                }
            }
        } else if (learn_func == 2) {
            //3.ART2
            double top, bottom;
            for (int j = 0; j < numCode + 1; j++) {
                T[j] = 0;
                for (int k = 0; k < numSpace; k++) {
                    top = 0;
                    bottom = 0;
                    for (int i = 0; i < numInput[k]; i++) { //(I v W) / W
                        top += I[k][i] * W[j][k][i];
                    }
                    bottom = distance(numInput[k], I[k]) * distance(numInput[k], W[j][k]);
                    T[j] += gamma[k] * (double) (top / bottom); //Tj
                }
            }
        } else if (learn_func == 3) {
            //4.comvinatorial operations
            double top, bottom;
            for (int j = 0; j < numCode + 1; j++) {//激活函数
                T[j] = 0;
                int k = 0;
                for (k = 0; k < REWARD; k++) {
                    top = 0;
                    bottom = (double) alpha[k];
                    for (int i = 0; i < numInput[k]; i++) { //(I v W) / W
                        top += Math.min(I[k][i], W[j][k][i]);
                        bottom += W[j][k][i];
                    }
                    T[j] += gamma[k] * (double) (top / bottom); //Tj
                }

                k = REWARD;
                top = 0;
                bottom = 0;
                for (int i = 0; i < numInput[k]; i++) { //(I v W) / W
                    top += I[k][i] * W[j][k][i];
                }
                bottom = distance(numInput[k], I[k]) * distance(numInput[k], W[j][k]);
                T[j] += gamma[k] * (double) (top / bottom); //Tj
            }
        }
    }


    public void target_CodeActivition(int numSpace) {
        if (learn_func == 0) {
            //1.ART2-A
            for (int j = 0; j < target_numCode + 1; j++) {//激活函数
                T[j] = 0;
                for (int k = 0; k < numSpace; k++) {
                    double sum = 0.0;
                    for (int i = 0; i < numInput[k]; i++) {
                        sum += Math.pow(I[k][i], 2);
                    }
                    sum = Math.pow(sum, 1.0 / 2.0);

                    double sum2 = 0.0;
                    for (int i = 0; i < numInput[k]; i++) {
                        if (I[k][i] <= alpha[k]) {
                            I[k][i] = 0;
                        } else {
                            I[k][i] /= sum;
                        }
                        sum2 += I[k][i];
                    }
                    sum2 = Math.pow(sum2, 1.0 / 2.0);

                    for (int i = 0; i < numInput[k]; i++) {
                        I[k][i] /= sum2;
                        if (newCode[j]) {
                            T[j] += alpha[k] * I[k][i];
                        } else {
                            T[j] += gamma[k] * I[k][i] * target_W[j][k][i];
                        }
                    }
                }
            }
        } else if (learn_func == 1) {

            //2.FuzzyART    	
            double top, bottom;
            for (int j = 0; j < target_numCode + 1; j++) {//激活函数
                T[j] = 0;
                for (int k = 0; k < numSpace; k++) {
                    top = 0;
                    bottom = (double) alpha[k];
                    for (int i = 0; i < numInput[k]; i++) { //(I v target_W) / W
                        top += Math.min(I[k][i], target_W[j][k][i]);
                        bottom += target_W[j][k][i];
                    }
                    T[j] += gamma[k] * (double) (top / bottom); //Tj
                }
            }
        } else if (learn_func == 2) {
            //3.ART2
            double top, bottom;
            for (int j = 0; j < target_numCode + 1; j++) {
                T[j] = 0;
                for (int k = 0; k < numSpace; k++) {
                    top = 0;
                    bottom = 0;
                    for (int i = 0; i < numInput[k]; i++) { //(I v W) / W
                        top += I[k][i] * target_W[j][k][i];
                    }
                    bottom = distance(numInput[k], I[k]) * distance(numInput[k], target_W[j][k]);
                    T[j] += gamma[k] * (double) (top / bottom); //Tj
                }
            }
        } else if (learn_func == 3) {
            //4.comvinatorial operations
            double top, bottom;
            for (int j = 0; j < target_numCode + 1; j++) {//激活函数
                T[j] = 0;
                int k = 0;
                for (k = 0; k < REWARD; k++) {
                    top = 0;
                    bottom = (double) alpha[k];
                    for (int i = 0; i < numInput[k]; i++) { //(I v target_W) / W
                        top += Math.min(I[k][i], target_W[j][k][i]);
                        bottom += target_W[j][k][i];
                    }
                    T[j] += gamma[k] * (double) (top / bottom); //Tj
                }
                top = 0;
                bottom = 0;
                k = REWARD;
                for (int i = 0; i < numInput[k]; i++) { //(I v W) / W
                    top += I[k][i] * target_W[j][k][i];
                }
                bottom = distance(numInput[k], I[k]) * distance(numInput[k], target_W[j][k]);
                T[j] += gamma[k] * (double) (top / bottom); //Tj
            }
        }
    }

    //代码竞争（判断最好的思考）

    public int CodeCompetition() {
        int J = 0;
        double max_act = (double) -Double.MAX_VALUE;
        for (int j = 0; j < numCode + 1; j++)
            if (T[j] > max_act) {
                max_act = T[j];
                J = j;
            }
        y[J] += 1;
        return J;
    }

    public int target_CodeCompetition() {
        int J = 0;
        double max_act = (double) -Double.MAX_VALUE;
        for (int j = 0; j < target_numCode + 1; j++)
            if (T[j] > max_act) {
                max_act = T[j];
                J = j;
            }
        return J;
    }

    //活动读出（提取记忆）
    public void WtoI(int J) {
        for (int i = 0; i < numInput[REWARD]; i++)
            I[REWARD][i] = W[J][REWARD][i];

    }

    public void target_WtoI(int J) {
        for (int i = 0; i < numInput[REWARD]; i++)
            I[REWARD][i] = target_W[J][REWARD][i];

    }

    public double Q_Function(double[] stat, double[] act) {
        double QValue = (double) 0.0;

        setState(stat);
        setAction(act);

        CodeActivition(2);            //代码激活（每个节点思考）
        int J = CodeCompetition();
        //从记忆中提取Q值

        WtoI(J);//F1节点提取记忆（Q值）
        if (I[REWARD][0] == I[REWARD][1] && I[REWARD][0] == 1) { //初次记忆
            QValue = (double) 0.5;
        } else
            QValue = I[REWARD][0];//有记忆就用记忆
        return (QValue);
    }

    //Q'
    public double target_Q_Function(double[] stat, double[] act) {
        setState(stat);
        setAction(act);
        double QValue = (double) 0.0;

        target_CodeActivition(2);            //代码激活（每个节点思考）
        int J = target_CodeCompetition();


        //从记忆中提取Q值
        target_WtoI(J);//F1节点提取记忆（Q值）
        if (I[REWARD][0] == I[REWARD][1] && I[REWARD][0] == 1) { //初次记忆
            QValue = (double) 0.5;
        } else
            QValue = I[REWARD][0];//有记忆就用记忆
        return (QValue);
    }


    public double eval(Buffer buff) {
        double y;
        if (buff.end == true) {
            y = buff.r;
        } else {
            y = MaxQValue(buff.next_stat, buff.r); //计算下次状态最大Q值
        }
        double[] act = new double[numAction];
        actiontoact(buff.action, act);
        double QValue = Q_Function(buff.stat, act);//计算本状态Q
        return Math.abs(y - QValue);
    }

    //行动策略选择行为
    public static double QEpsilonDecay = (double) 0.00050;
    public static double QEpsilon = (double) 0.50000;

    public int access = 1;

    public int Policy(double[] stat, boolean[] vact) {
        double[] qValues = new double[numAction];
        int selectedAction = -1;
        double maxQ = -Double.MAX_VALUE;

        //策略选择行为
        if (Math.random() < QEpsilon) {//随机行为
            while (true) {
                int randomIndex = (int) (Math.random() * numAction);
                selectedAction = randomIndex;
                if (vact[selectedAction]) {
                    break;
                }
            }
            return selectedAction;
        } else {//选择的行为
            if (access == 0) {
                //Direct Access
                setState(stat);
                for (int i = 0; i < numInput[ACTION]; i++)
                    if (vact[i] == true) {
                        I[ACTION][i] = 1;
                    } else {
                        I[ACTION][i] = 0;
                    }
                setReward(1);

                CodeActivition(3);            //代码激活（每个节点思考）
                int J = CodeCompetition();
                for (int i = 0; i < numInput[ACTION]; i++) {
                    I[ACTION][i] = Math.min(I[ACTION][i], W[J][ACTION][i]);
                    if (maxQ < I[ACTION][i]) {
                        maxQ = I[ACTION][i];
                        selectedAction = i;
                    }
                }
                return selectedAction;

            } else if (access == 1) {
                //valliance
                int[] doubleValues = new int[qValues.length];
                int maxDV = 0;

                for (int i = 0; i < numAction; i++) {

                    int action = i;
                    double[] act = new double[numAction];
                    actiontoact(action, act);
                    if (vact[i] == true) {
                        qValues[i] = Q_Function(stat, act);   //估计当前状态下所有的行为Q值
                    } else {
                        qValues[i] = -Double.MAX_VALUE;
                    }
                    if (qValues[action] > maxQ) {
                        maxDV = 0;
                        selectedAction = action;
                        maxQ = qValues[action];
                        doubleValues[maxDV] = selectedAction;
                        maxDV = 1;
                    } else if (qValues[action] == maxQ) {
                        doubleValues[maxDV] = action;
                        maxDV++;
                    }
                }

                if (maxDV > 1) {   // 如果最好行为重复了，随机选一个
                    int randomIndex = (int) (Math.random() * maxDV);
                    selectedAction = doubleValues[randomIndex];
                }
            }
            return selectedAction;
        }

    }


/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////


    //学习部分

    //模板匹配（判断是否合理）
    public double TemplateMatching(int k, int j) {
        double m = (double) 0.0;
        double denominator = (double) 0.0; //分母

        for (int i = 0; i < numInput[k]; i++) {//相似度
            m += Math.min(I[k][i], W[j][k][i]);
            denominator += I[k][i];
        }

        if (denominator == 0)
            return (1);
        else
            return (m / denominator);
    }

    //模板学习（更新记忆）
    public double beta = (double) 0.1;

    public void TemplateLearning(int J) {
        if (newCode[J]) {
            if (learn_func == 1) {
                //fuzzy-ART
                for (int k = 0; k < numSpace; k++) {//再次记忆
                    for (int i = 0; i < numInput[k]; i++) {
                        W[J][k][i] = Math.min(W[J][k][i], I[k][i]);
                    }
                }
            } else if (learn_func == 2) {
                //ART2
                for (int k = 0; k < numSpace; k++) {//再次记忆
                    for (int i = 0; i < numInput[k]; i++) {
                        W[J][k][i] = I[k][i];

                    }
                }
            } else if (learn_func == 3) {
                //combinatorial operations
                int k = 0;
                for (k = 0; k < REWARD; k++) {//再次记忆
                    for (int i = 0; i < numInput[k]; i++) {
                        W[J][k][i] = Math.min(W[J][k][i], I[k][i]);

                    }
                }
                k = REWARD;
                for (int i = 0; i < numInput[k]; i++) {
                    W[J][k][i] = I[k][i];
                }
                k = NEXTSTATE;
                for (int i = 0; i < numInput[k]; i++) {
                    W[J][k][i] = Math.min(W[J][k][i], I[k][i]);

                }
            }
            newCode[J] = false;
            createNewCode();//创建新的节点
        } else {
            if (learn_func == 1) {
                //fuzzy-ART
                for (int k = 0; k < numSpace; k++) {//再次记忆
                    for (int i = 0; i < numInput[k]; i++) {
                        W[J][k][i] = (1 - beta) * W[J][k][i] + beta * Math.min(W[J][k][i], I[k][i]);                        //W = (1- b)w + b w*I
                        //W[J][k][i] = Math.min(W[J][k][i] , ( W[J][k][i]- beta * (W[J][k][i]-I[k][i]) ) );

                    }
                }
            } else if (learn_func == 2) {
                //ART2
                for (int k = 0; k < numSpace; k++) {//再次记忆
                    for (int i = 0; i < numInput[k]; i++) {
                        W[J][k][i] = (1 - beta) * W[J][k][i] + beta * I[k][i];

                    }
                }
            } else if (learn_func == 3) {
                //combinatorial operations
                int k = 0;
                for (k = 0; k < REWARD; k++) {//再次记忆
                    for (int i = 0; i < numInput[k]; i++) {
                        W[J][k][i] = (1 - beta) * W[J][k][i] + beta * Math.min(W[J][k][i], I[k][i]);

                    }
                }
                k = REWARD;
                for (int i = 0; i < numInput[k]; i++) {
                    W[J][k][i] = (1 - beta) * W[J][k][i] + beta * I[k][i];
                }
                k = NEXTSTATE;
                for (int i = 0; i < numInput[k]; i++) {
                    W[J][k][i] = (1 - beta) * W[J][k][i] + beta * Math.min(W[J][k][i], I[k][i]);

                }

            }
        }
    }

    //新写入

    public void doOverwrite(int J) {
        for (int k = 0; k < numSpace; k++)
            for (int i = 0; i < numInput[k]; i++)
                W[J][k][i] = I[k][i];
    }

    //创建新节点    
    public void createNewCode() {

        if (numCode < capacity)
            numCode++;
        newCode[numCode] = true;

        W[numCode] = new double[numSpace][];
        for (int k = 0; k < numSpace; k++) {
            W[numCode][k] = new double[numInput[k]];
            for (int i = 0; i < numInput[k]; i++)
                W[numCode][k][i] = (double) 1.0;
        }
    }

    //计算下一状态的softmax Q值
    //softmax Q'(s' , a')= log  [< (e**Q'(s',a') )/N
    public double QGamma = (double) 0.1;

    public double SoftMaxQValue(double[] next_stat, double r) {
        double softmax_Q = -Double.MAX_VALUE;
        double sum_soft_Q = 0.0;
        for (int i = 0; i < numAction; i++) {//Q-学习
            int action = i;
            double[] act = new double[numAction];
            actiontoact(action, act);
            //double tmp_Q = target_Q_Function(next_stat,act);
            double tmp_Q = Q_Function(next_stat, act);
            sum_soft_Q += Math.pow(Math.E, tmp_Q);
        }
        sum_soft_Q /= numAction;
        softmax_Q = Math.log(sum_soft_Q);
        double y = r + QGamma * softmax_Q;
        return y;
    }


    public double MaxQValue(double[] next_stat, double r) {
        double max_Q = -Double.MAX_VALUE;
        double sum_soft_Q = 0.0;
        for (int i = 0; i < numAction; i++) {//Q-学习
            int action = i;
            double[] act = new double[numAction];
            actiontoact(action, act);
            //double tmp_Q = target_Q_Function(next_stat,act);
            double tmp_Q = Q_Function(next_stat, act);
            if (max_Q < tmp_Q) {
                max_Q = tmp_Q;
            }
        }
        double y = r + QGamma * max_Q;
        return y;
    }

    //Q学习公式
    public double QAlpha = (double) 0.5;

    //Q( s , a ) = Q( s , a )- a(r(s,a) + softmax Q' (s',a') ) )
    public double QValue_Update(double QValue, double y) {
        QValue = QValue - QAlpha * (QValue - y);// 新Q = 旧Q +（奖励 + 新状态最大Q - 旧Q）
        if (QValue < 0) QValue = 0;
        if (QValue > 1) QValue = 1;
        return QValue;
    }


    public double epilson = (double) 0.001;
    double rho[] = {(double) 0.2, (double) 0.2, (double) 0.5};
    double[] match = new double[numSpace];

    public void Q_Function_update(Buffer buff, double[] p) {
        int numSpace = 3;

        for (int i = 0; i < numSpace; i++) {
            rho[i] = p[i];
        }
        double y;
        if (buff.end == true) {
            y = buff.r;
        } else {
            y = MaxQValue(buff.next_stat, buff.r); //计算下次状态最大Q值
            //y=MaxQValue(buff.next_stat,buff.r);
        }

        //if (y<0) y = 0;
        //if (y>1) y = 1;

        double[] act = new double[numAction];
        actiontoact(buff.action, act);
        double QValue = Q_Function(buff.stat, act);//计算本状态Q
        buff.p = Math.abs(y - QValue);
        // buff.p=Math.abs(buff.r+al);
        //System.out.println(buff.w);
        //System.out.println(buff.p);

        QValue = QValue_Update(QValue, y);//Q学习公式
        //代理读取之前的状态

        setState(buff.stat);
        actiontoact(buff.action, act);
        setAction(act);
        setReward(QValue);
        setNextState(buff.next_stat);
        //buff.System.out.println(QValue);

        boolean reset = true,
                perfectMismatch = false;

        CodeActivition(3);            //代码激活（每个节点思考）
        int J = CodeCompetition();        //代码竞争（判断最好的思考）
        //模板学习（再次记忆）
        while (reset && !perfectMismatch) {
            reset = false;
            for (int k = 0; k < numSpace; k++)
                match[k] = TemplateMatching(k, J);    //模板匹配（判断是否合理）
            if (match[CURSTATE] < rho[CURSTATE] || match[ACTION] < rho[ACTION] || match[REWARD] < rho[REWARD]) {//如果相似度过低
                if (match[CURSTATE] == 1) {//状态相似度完全相同
                    perfectMismatch = true;
                } else {//相似度低就重新初始化并更新阈值函数并重新选择
                    T[J] = -Double.MAX_VALUE;
                    reset = true;
                    for (int k = 0; k < 1; k++)
                        if (match[k] > rho[k]) {
                            rho[k] = Math.min(match[k] + epilson, 1);
                            p[k] = rho[k];
                        }
                    J = CodeCompetition(); //代码竞争（判断最好的思考）
                }
            }
        }

        if (perfectMismatch)
            doOverwrite(J);
        else
            TemplateLearning(J);
    }


    public void target_Q_Function_update() {
        target_numCode = numCode;

        //System.out.println("numCode");
        //System.out.println(numCode);
        for (int j = 0; j < target_numCode + 1; j++) {
            target_W[j] = new double[numSpace][];
            for (int k = 0; k < numSpace; k++) {
                target_W[j][k] = new double[numInput[k]];
                for (int i = 0; i < numInput[k]; i++) {
                    target_W[j][k][i] = W[j][k][i];
                }
            }
        }
    }


///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////


    public NN nnet;

    public void WtoA(int J) {
        for (int i = 0; i < numInput[ACTION]; i++)
            I[ACTION][i] = W[J][ACTION][i];

    }

    public int Policy2(double[] stat) {
        double[] pol = new double[numAction];
        pol = nnet.policy(stat);

        //随机选一个动作
        double rand = Math.random();
        int j = 0;
        for (int i = 0; i < numAction; i++) {
            rand -= pol[i];
            if (rand > 0) {
                j++;
            } else {
                break;
            }
        }
        return j;

    }


/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////


    public double VAlpha = (double) 0.5;

    public double getVValue(double[] next_stat, double[] pol) {
        double VValue = 0.0;
        for (int i = 0; i < numAction; i++) {//Q-学习
            int action = i;
            double[] act = new double[numAction];
            actiontoact(action, act);
            VValue += Q_Function(next_stat, act) * pol[i];
        }
        return VValue;
    }

    public void Q_Function_update2(Buffer buff, double[] p) {
        int numSpace = 3;
        double rho[] = {(double) 0.2, (double) 0.2, (double) 0.5};
        for (int i = 0; i < numSpace; i++) {
            rho[i] = p[i];
        }
        double y;
        if (buff.end == true) {
            y = buff.r;
        } else {
            y = SoftMaxQValue(buff.next_stat, buff.r); //计算下次状态最大Q值
        }

        double[] act = new double[numAction];
        actiontoact(buff.action, act);
        double QValue = Q_Function(buff.stat, act);//计算本状态Q
        buff.p = Math.abs(y - QValue);
        QValue = QValue_Update(QValue, y);//Q学习公式


        setState(buff.stat);
        actiontoact(buff.action, act);
        setAction(act);
        setReward(QValue);

        boolean reset = true,
                perfectMismatch = false;

        CodeActivition(3);            //代码激活（每个节点思考）
        int J = CodeCompetition();        //代码竞争（判断最好的思考）

        //模板学习（再次记忆）
        while (reset && !perfectMismatch) {
            reset = false;
            for (int k = 0; k < numSpace; k++)
                match[k] = TemplateMatching(k, J);    //模板匹配（判断是否合理）
            if (match[CURSTATE] < rho[CURSTATE] || match[ACTION] < rho[ACTION] || match[REWARD] < rho[REWARD]) {//如果相似度过低
                if (match[CURSTATE] == 1) {//状态相似度完全相同
                    perfectMismatch = true;
                } else {//相似度低就重新初始化并更新阈值函数并重新选择
                    T[J] = -Double.MAX_VALUE;
                    reset = true;
                    for (int k = 0; k < 1; k++)
                        if (match[k] > rho[k])
                            rho[k] = Math.min(match[k] + epilson, 1);
                    J = CodeCompetition(); //代码竞争（判断最好的思考）
                }
            }
        }
        if (perfectMismatch)
            doOverwrite(J);
        else
            TemplateLearning(J);

        if (buff.end == true) {
            y = buff.r;
        } else {
            y = buff.r + getVValue(buff.next_stat, nnet.policy(buff.next_stat));
        }
        double VValue = getVValue(buff.stat, nnet.policy(buff.stat));
        double AValue = y - VValue;
        nnet.learn(buff.stat, buff.action, AValue);

    }


    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////


    public void readfile(int agent_num) throws IOException {
        String path = "e:/store-" + agent_num + ".txt";
        FileInputStream in = new FileInputStream(path);
        DataInputStream din = new DataInputStream(in);
        numCode = din.readInt();
        target_numCode = numCode;

        newCode = new boolean[capacity];
        y = new int[capacity];


        T = new double[capacity];

        W = new double[capacity][][];
        target_W = new double[capacity][][];


        for (int j = 0; j < numCode + 1; j++) {

            W[j] = new double[numSpace][];
            newCode[j] = din.readBoolean();

            y[j] = din.readInt();
            for (int k = 0; k < numSpace; k++) {

                W[j][k] = new double[numInput[k]];
                for (int i = 0; i < numInput[k]; i++) {
                    W[j][k][i] = din.readDouble();
                }
            }
        }
        din.close();
        in.close();

    }

    public void writefile(int agent_num) throws PogamutException, Exception {
        String path = "e:/store-" + agent_num + ".txt";
        FileOutputStream out = new FileOutputStream(path);
        DataOutputStream dout = new DataOutputStream(out);

        dout.writeInt(numCode);

        for (int j = 0; j < numCode + 1; j++) {

            dout.writeBoolean(newCode[j]);
            dout.writeInt(y[j]);

            for (int k = 0; k < numSpace; k++) {
                for (int i = 0; i < numInput[k]; i++) {
                    dout.writeDouble(W[j][k][i]);
                }
            }
        }
        dout.close();
        out.close();
    }


}